# Scrapy settings for scrapycompositedemo project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = "scrapycompositedemo"

SPIDER_MODULES = ["scrapycompositedemo.spiders"]
NEWSPIDER_MODULE = "scrapycompositedemo.spiders"

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

RETRY_HTTP_CODES = [401, 403, 500, 502, 503, 504]
CONCURRENT_REQUESTS = 10
DOWNLOAD_TIMEOUT = 10
RETRY_TIMES = 10

# Set settings whose default value is deprecated to a future-proof value
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
FEED_EXPORT_ENCODING = "utf-8"


DOWNLOADER_MIDDLEWARES = {
    'scrapycompositedemo.middlewares.AuthorizationMiddleware': 543,
    'scrapycompositedemo.middlewares.ProxyMiddleware': 544,
}

GENeRATOR_MAP = {
    'antispider7': 'Antispider7Generator'
}
TESTER_MAP = {
    'antispider7': 'Antispider7Tester'
}
TEST_URL_MAP = {
    'antispider7': 'https://antispider7.scrape.center/api/book/?limit=18&offset=0'
}

